{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "55000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/william/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py:1714: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 0.336292\n",
      "0.9049\n",
      "step 200, entropy loss: 0.289314\n",
      "0.9181\n",
      "step 300, entropy loss: 0.271351\n",
      "0.9423\n",
      "step 400, entropy loss: 0.356520\n",
      "0.9433\n",
      "step 500, entropy loss: 0.274182\n",
      "0.9535\n",
      "step 600, entropy loss: 0.207656\n",
      "0.9538\n",
      "step 700, entropy loss: 0.111825\n",
      "0.9639\n",
      "step 800, entropy loss: 0.091928\n",
      "0.9622\n",
      "step 900, entropy loss: 0.099280\n",
      "0.9619\n",
      "step 1000, entropy loss: 0.134722\n",
      "0.964\n",
      "step 1100, entropy loss: 0.063684\n",
      "0.9694\n",
      "step 1200, entropy loss: 0.090447\n",
      "0.9708\n",
      "step 1300, entropy loss: 0.094965\n",
      "0.9682\n",
      "step 1400, entropy loss: 0.124732\n",
      "0.9691\n",
      "step 1500, entropy loss: 0.062899\n",
      "0.9693\n",
      "step 1600, entropy loss: 0.119884\n",
      "0.9671\n",
      "step 1700, entropy loss: 0.053609\n",
      "0.9723\n",
      "step 1800, entropy loss: 0.109073\n",
      "0.9717\n",
      "step 1900, entropy loss: 0.103306\n",
      "0.9712\n",
      "step 2000, entropy loss: 0.119701\n",
      "0.972\n",
      "step 2100, entropy loss: 0.076628\n",
      "0.9691\n",
      "step 2200, entropy loss: 0.046256\n",
      "0.9741\n",
      "step 2300, entropy loss: 0.047919\n",
      "0.9716\n",
      "step 2400, entropy loss: 0.230844\n",
      "0.9724\n",
      "step 2500, entropy loss: 0.042556\n",
      "0.9749\n",
      "step 2600, entropy loss: 0.029623\n",
      "0.976\n",
      "step 2700, entropy loss: 0.013118\n",
      "0.9737\n",
      "step 2800, entropy loss: 0.018236\n",
      "0.9772\n",
      "step 2900, entropy loss: 0.091970\n",
      "0.9751\n",
      "step 3000, entropy loss: 0.048863\n",
      "0.9767\n",
      "step 3100, entropy loss: 0.083386\n",
      "0.9765\n",
      "step 3200, entropy loss: 0.026580\n",
      "0.9758\n",
      "step 3300, entropy loss: 0.019094\n",
      "0.977\n",
      "step 3400, entropy loss: 0.010859\n",
      "0.977\n",
      "step 3500, entropy loss: 0.038979\n",
      "0.9776\n",
      "step 3600, entropy loss: 0.020836\n",
      "0.9759\n",
      "step 3700, entropy loss: 0.047455\n",
      "0.9777\n",
      "step 3800, entropy loss: 0.029084\n",
      "0.977\n",
      "step 3900, entropy loss: 0.012936\n",
      "0.9781\n",
      "step 4000, entropy loss: 0.034824\n",
      "0.9784\n",
      "step 4100, entropy loss: 0.027768\n",
      "0.9771\n",
      "step 4200, entropy loss: 0.018154\n",
      "0.9773\n",
      "step 4300, entropy loss: 0.029935\n",
      "0.9795\n",
      "step 4400, entropy loss: 0.028758\n",
      "0.9769\n",
      "step 4500, entropy loss: 0.010151\n",
      "0.9789\n",
      "step 4600, entropy loss: 0.021763\n",
      "0.9787\n",
      "step 4700, entropy loss: 0.010732\n",
      "0.9782\n",
      "step 4800, entropy loss: 0.008551\n",
      "0.9759\n",
      "step 4900, entropy loss: 0.014345\n",
      "0.9778\n",
      "step 5000, entropy loss: 0.006288\n",
      "0.9783\n",
      "step 5100, entropy loss: 0.010605\n",
      "0.977\n",
      "step 5200, entropy loss: 0.030162\n",
      "0.9791\n",
      "step 5300, entropy loss: 0.016926\n",
      "0.9788\n",
      "step 5400, entropy loss: 0.093978\n",
      "0.9753\n",
      "step 5500, entropy loss: 0.043325\n",
      "0.9779\n",
      "step 5600, entropy loss: 0.040864\n",
      "0.9764\n",
      "step 5700, entropy loss: 0.015333\n",
      "0.979\n",
      "step 5800, entropy loss: 0.028151\n",
      "0.9778\n",
      "step 5900, entropy loss: 0.010998\n",
      "0.9791\n",
      "step 6000, entropy loss: 0.028074\n",
      "0.9793\n",
      "step 6100, entropy loss: 0.015051\n",
      "0.9789\n",
      "step 6200, entropy loss: 0.012156\n",
      "0.9798\n",
      "step 6300, entropy loss: 0.013834\n",
      "0.9784\n",
      "step 6400, entropy loss: 0.018822\n",
      "0.9775\n",
      "step 6500, entropy loss: 0.003866\n",
      "0.9771\n",
      "step 6600, entropy loss: 0.014101\n",
      "0.9805\n",
      "step 6700, entropy loss: 0.004379\n",
      "0.98\n",
      "step 6800, entropy loss: 0.009547\n",
      "0.9775\n",
      "step 6900, entropy loss: 0.010269\n",
      "0.9787\n",
      "step 7000, entropy loss: 0.014217\n",
      "0.978\n",
      "step 7100, entropy loss: 0.009895\n",
      "0.979\n",
      "step 7200, entropy loss: 0.109327\n",
      "0.9779\n",
      "step 7300, entropy loss: 0.004268\n",
      "0.9798\n",
      "step 7400, entropy loss: 0.013704\n",
      "0.9765\n",
      "step 7500, entropy loss: 0.015085\n",
      "0.9798\n",
      "step 7600, entropy loss: 0.005087\n",
      "0.9796\n",
      "step 7700, entropy loss: 0.005009\n",
      "0.9773\n",
      "step 7800, entropy loss: 0.010547\n",
      "0.9796\n",
      "step 7900, entropy loss: 0.003743\n",
      "0.9795\n",
      "step 8000, entropy loss: 0.006637\n",
      "0.9799\n",
      "step 8100, entropy loss: 0.010187\n",
      "0.9798\n",
      "step 8200, entropy loss: 0.010087\n",
      "0.9795\n",
      "step 8300, entropy loss: 0.005944\n",
      "0.9799\n",
      "step 8400, entropy loss: 0.004831\n",
      "0.98\n",
      "step 8500, entropy loss: 0.004674\n",
      "0.9806\n",
      "step 8600, entropy loss: 0.003167\n",
      "0.9797\n",
      "step 8700, entropy loss: 0.003533\n",
      "0.9803\n",
      "step 8800, entropy loss: 0.011227\n",
      "0.98\n",
      "step 8900, entropy loss: 0.004482\n",
      "0.9806\n",
      "step 9000, entropy loss: 0.002500\n",
      "0.9801\n"
     ]
    }
   ],
   "source": [
    "# 初学者参考文档 \n",
    "# MNIST For ML Beginners\n",
    "# https://www.tensorflow.org/versions/r1.0/get_started/mnist/beginners\n",
    "\n",
    "# 初学者参考代码地址\n",
    "# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist_softmax.py\n",
    "\n",
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "def swish(x):\n",
    "  return x * tf.nn.sigmoid(x)\n",
    "\n",
    "\n",
    "def selu(x):\n",
    "  with tf.name_scope('elu') as scope:\n",
    "    alpha = 1.6732632423543772848170429916717\n",
    "    scale = 1.0507009873554804934193349852946\n",
    "    return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def activation(x):\n",
    "#  return selu(x)\n",
    "    return relu(x)\n",
    "#  return tf.nn.sigmoid(x)\n",
    "#  return tf.nn.elu(x)\n",
    "#   return swish(x)\n",
    "#     return tf.nn.tanh(x)\n",
    "    \n",
    "def initialize(shape, stddev=0.1):\n",
    "  # 从截断的正态分布中输出随机值。\n",
    "  # 在正态分布的曲线中，横轴区间（μ-σ，μ+σ）内的面积为68.268949%。 \n",
    "  # 横轴区间（μ-2σ，μ+2σ）内的面积为95.449974%。\n",
    "  # 横轴区间（μ-3σ，μ+3σ）内的面积为99.730020%。 \n",
    "  # 在tf.truncated_normal中如果x的取值在区间（μ-2σ，μ+2σ）之外则重新进行选择。这样保证了生成的值都在均值附近。\n",
    "  # shape: 一维的张量，也是输出的张量。\n",
    "  # mean: 正态分布的均值。\n",
    "  # stddev: 正态分布的标准差。\n",
    "  # dtype: 输出的类型。\n",
    "  # seed: 一个整数，当设置之后，每次生成的随机数都一样。\n",
    "  # name: 操作的名字。\n",
    "  return tf.truncated_normal(shape, stddev=stddev)\n",
    "  #return tf.zeros(shape)\n",
    "\n",
    "\n",
    "# Import data\n",
    "# data_dir = '../data/mnist/input_data/'\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "print(mnist.train.num_examples)\n",
    "\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "# Create the model\n",
    "L1_units_count = 100\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "#tf.shape(x)  [100, 784]\n",
    "# exponetial learning rate decay\n",
    "epoch_steps = tf.to_int64(tf.div(60000, tf.shape(x)[0]))  # 1 epoch = 600 steps\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "# 1.单斜杠（/）表示除法，且不管除数和被除数是不是整数，最后结果都是float类型。\n",
    "# 2.双斜杠（//）表示地板除，即先做除法（/），然后向下取整（floor）。\n",
    "# 至少有一方是float型时，结果为float型；两个数都是int型时，结果为int型。\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch \n",
    "current_learning_rate = tf.multiply(init_learning_rate, \n",
    "                                    tf.pow(0.575, tf.to_float(decay_times)))\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count], \n",
    "                             stddev=np.sqrt(2/784)))\n",
    "b_1 = tf.Variable(tf.constant(0.001, shape=[L1_units_count])) # 1 X 100\n",
    "logits_1 = tf.matmul(x, W_1) + b_1\n",
    "output_1 = activation(logits_1)\n",
    "\n",
    "L2_units_count = 10 \n",
    "W_2 = tf.Variable(initialize([L1_units_count, \n",
    "                              L2_units_count], \n",
    "                             stddev=np.sqrt(2/L1_units_count)))\n",
    "b_2 = tf.Variable(tf.constant(0.001, shape=[L2_units_count])) # 1 X 10\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "\n",
    "y = logits_2\n",
    "\n",
    "\n",
    "\n",
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "# Teacher Code 1\n",
    "# l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2)\n",
    "# total_loss = cross_entropy + 4e-5*l2_loss\n",
    "\n",
    "# optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "# gradients = optimizer.compute_gradients(total_loss)\n",
    "# train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "# train_step = tf.train.AdamOptimizer(\n",
    "#     current_learning_rate).minimize(\n",
    "#     total_loss, global_step=global_step)\n",
    "\n",
    "################ start ##################\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy, global_step=global_step)\n",
    "################ end   ##################\n",
    "\n",
    "# weight decay      \n",
    "#0 0 0 0 1 0 0 0 0\n",
    "#0 1 0 0 0 0 0 0 0\n",
    "# 还记得不？不要把graph定义写到图里面\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "# Teacher Code 2\n",
    "# Train\n",
    "# for step in range(3000):\n",
    "#   batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "#   lr = 1e-2\n",
    "#   _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "#       sess.run(\n",
    "#                [train_step, cross_entropy, l2_loss, total_loss, \n",
    "#                 current_learning_rate], \n",
    "#                feed_dict={x: batch_xs, y_: batch_ys, \n",
    "#                           init_learning_rate:lr})\n",
    "  \n",
    "#   if (step+1) % 100 == 0:\n",
    "#     print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "#             (step+1, loss, l2_loss_value, total_loss_value))\n",
    "#     print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "#                                     y_: mnist.test.labels}))\n",
    "\n",
    "################ start ##################    \n",
    "# Train\n",
    "for step in range(9000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    _, loss = sess.run([train_step, cross_entropy], \n",
    "             feed_dict={x: batch_xs, y_: batch_ys})\n",
    "\n",
    "# Test trained model \n",
    "    if (step+1) % 100 == 0:\n",
    "        print('step %d, entropy loss: %f' % \n",
    "            (step+1, loss))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "            y_: mnist.test.labels}))\n",
    "################ end #####################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}